{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "321dd425",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import shutil\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import onekey_algo.custom.components as okcomp\n",
    "from onekey_algo import get_param_in_cwd\n",
    "\n",
    "plt.rcParams['figure.dpi'] = 300\n",
    "model_names = get_param_in_cwd('compare_models')\n",
    "# 获取配置\n",
    "task = get_param_in_cwd('task_column')\n",
    "sel_m = get_param_in_cwd('sel_model')\n",
    "labelf = get_param_in_cwd('label_file') or os.path.join(mydir, 'label.csv')\n",
    "group_info = get_param_in_cwd('dataset_column') or 'group'\n",
    "\n",
    "# 读取label文件。\n",
    "labels = [task]\n",
    "label_data_ = pd.read_csv(labelf)\n",
    "label_data_['ID'] = label_data_['ID'].map(lambda x: f\"{x}.nii.gz\" if not (f\"{x}\".endswith('.nii.gz') or  f\"{x}\".endswith('.nii')) else x)\n",
    "label_data_ = label_data_[['ID', group_info, task]]\n",
    "label_data_ = label_data_.dropna(axis=0)\n",
    "\n",
    "ids = label_data_['ID']\n",
    "print(label_data_.columns)\n",
    "label_data = label_data_\n",
    "label_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac5ee2cf",
   "metadata": {},
   "source": [
    "# 训练集-Nomogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5002786",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from onekey_algo.custom.components.comp1 import normalize_df, merge_results\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from onekey_algo.custom.components import metrics\n",
    "from onekey_algo.custom.components.delong import delong_roc_test\n",
    "from onekey_algo.custom.components.comp1 import draw_matrix\n",
    "from onekey_algo.custom.components.metrics import NRI, IDI\n",
    "from onekey_algo.custom.components.comp1 import plot_DCA\n",
    "from onekey_algo.custom.components.comp1 import draw_calibration\n",
    "from onekey_algo.custom.components import stats\n",
    "from onekey_algo.custom.components.metrics import analysis_pred_binary\n",
    "\n",
    "hosmer = []\n",
    "youden = {}\n",
    "metric = []\n",
    "fig_size = (5, 4)\n",
    "for subset in [s for s in get_param_in_cwd('subsets', ['train', 'test'])]:\n",
    "    ALL_results = None\n",
    "    for mn in  model_names:\n",
    "        r = pd.read_csv(f\"./results/{mn}_{sel_m[mn]}_{subset if subset == 'train' else 'test'}.csv\")\n",
    "        r.columns = ['ID', '-0', mn]\n",
    "        if ALL_results is None:\n",
    "            ALL_results = r\n",
    "        else:\n",
    "            ALL_results = pd.merge(ALL_results, r, on='ID', how='inner')\n",
    "    Clinic = pd.read_csv('data/clinic_sel.csv')\n",
    "    cnames = [c for c in Clinic.columns if c not in ['ID', 'group', 'label']]\n",
    "    ALL_results =merge_results(ALL_results, Clinic[['ID'] + cnames], label_data[label_data['group'] == subset], label_col='ID')\n",
    "    ALL_results = ALL_results.dropna(axis=1)\n",
    "    display(ALL_results.shape)\n",
    "    # 绘制整体的ROC曲线\n",
    "    pred_column = [f'{task}-0', f'{task}-1']\n",
    "    gt = [np.array(ALL_results[task]) for _ in model_names]\n",
    "    pred_train = [np.array(ALL_results[d]) for d in model_names]\n",
    "    okcomp.comp1.draw_roc(gt, pred_train, labels=model_names, title=f'Cohort {subset} ROC', auto_point=False)\n",
    "    plt.savefig(f'img/mc_{subset}_auc.svg')\n",
    "    plt.show()\n",
    "    \n",
    "    # 汇总所有的Metric\n",
    "    for mname, y, score in zip(model_names, gt, pred_train):\n",
    "        # 计算验证集指标\n",
    "        acc, auc, ci, tpr, tnr, ppv, npv, precision, recall, f1, thres = analysis_pred_binary(y, score)\n",
    "        ci = f\"{ci[0]:.4f} - {ci[1]:.4f}\"\n",
    "        youden[mname] = thres\n",
    "        metric.append((mname, acc, auc, ci, tpr, tnr, ppv, npv, precision, recall, f1, thres, subset))\n",
    "    metric_ = pd.DataFrame(metric, index=None, columns=['Signature', 'Accuracy', 'AUC', '95% CI',\n",
    "                                                       'Sensitivity', 'Specificity', \n",
    "                                                       'PPV', 'NPV', 'Precision', 'Recall', 'F1',\n",
    "                                                       'Threshold', 'Cohort'])\n",
    "\n",
    "    display(metric_)\n",
    "    \n",
    "    # 绘制Delong\n",
    "    delong = []\n",
    "    delong_columns = []\n",
    "    this_delong = []\n",
    "    plt.figure(figsize=fig_size)\n",
    "    cm = np.zeros((len(model_names), len(model_names)))\n",
    "    for i, mni in enumerate(model_names):\n",
    "        for j, mnj in enumerate(model_names):\n",
    "            if i <= j:\n",
    "                cm[i][j] = np.nan\n",
    "            else:\n",
    "                cm[i][j] = delong_roc_test(ALL_results[task], ALL_results[mni], ALL_results[mnj])[0][0]\n",
    "    cm = pd.DataFrame(cm[1:, :-1], index=model_names[1:], columns=model_names[:-1])\n",
    "    draw_matrix(cm, annot=True, cmap='jet_r', cbar=True)\n",
    "    plt.title(f'Cohort {subset} Delong')\n",
    "    plt.savefig(f'img/mc_delong_each_cohort_{subset}.svg', bbox_inches = 'tight')\n",
    "    plt.show()\n",
    "    \n",
    "    # NRI\n",
    "    delong = []\n",
    "    delong_columns = []\n",
    "    this_delong = []\n",
    "    plt.figure(figsize=fig_size)\n",
    "    cm = np.zeros((len(model_names), len(model_names)))\n",
    "    for i, mni in enumerate(model_names):\n",
    "        for j, mnj in enumerate(model_names):\n",
    "            cm[i][j] = NRI(ALL_results[mni] > youden[mni], ALL_results[mnj] > youden[mnj], ALL_results[task])\n",
    "    cm = pd.DataFrame(cm, index=model_names, columns=model_names)\n",
    "    draw_matrix(cm, annot=True, cmap='jet_r', cbar=True)\n",
    "    plt.title(f'Cohort {subset} NRI')\n",
    "    plt.savefig(f'img/mc_NRI_each_cohort_{subset}.svg', bbox_inches = 'tight')\n",
    "    plt.show()\n",
    "    \n",
    "    # IDI\n",
    "    delong = []\n",
    "    delong_columns = []\n",
    "    this_delong = []\n",
    "    cm = np.zeros((len(model_names), len(model_names)))\n",
    "    p = np.zeros((len(model_names), len(model_names)))\n",
    "    for i, mni in enumerate(model_names):\n",
    "        for j, mnj in enumerate(model_names):\n",
    "            cm[i][j], p[i][j] = IDI(ALL_results[mni], ALL_results[mnj], ALL_results[task], with_p=True)\n",
    "\n",
    "    for d, n in zip([cm, p], ['IDI', 'IDI pvalue']):\n",
    "        plt.figure(figsize=fig_size)\n",
    "        d = pd.DataFrame(d, index=model_names, columns=model_names)\n",
    "        draw_matrix(d, annot=True, cmap='jet_r', cbar=True)\n",
    "        plt.title(f'Cohort {subset} {n}')\n",
    "        plt.savefig(f'img/mc_{n}_each_cohort_{subset}.svg', bbox_inches = 'tight')\n",
    "        plt.show()\n",
    "        \n",
    "    # DCA\n",
    "    plot_DCA([ALL_results[model_name] for model_name in model_names], \n",
    "             ALL_results[task], title=f'Cohort {subset} DCA', labels=model_names, y_min=-0.15)\n",
    "    plt.savefig(f'img/mc_{subset}_dca.svg')\n",
    "    plt.show()\n",
    "    \n",
    "    # Calibration\n",
    "    draw_calibration(pred_scores=pred_train, n_bins=5,\n",
    "                     y_test=gt, model_names=model_names)\n",
    "    plt.title(f'Cohort {subset} Calibration')\n",
    "    plt.savefig(f'img/mc_{subset}_cali.svg')\n",
    "    plt.show()\n",
    "    \n",
    "    # HLTest\n",
    "    hosmer.append([stats.hosmer_lemeshow_test(y_true, y_pred, bins=10) \n",
    "                  for fn, y_true, y_pred in zip(model_names, gt, pred_train)])\n",
    "pd.concat([pd.DataFrame(hosmer, columns=model_names), pd.DataFrame(get_param_in_cwd('subsets'), columns=['Cohort'])], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "677780d5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
